In [ ]:
# making sure we use GPU1
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
import RARL_wm
import math
import numpy as np
import pandas as pd
import torch
from safety_rl.RARL.utils import save_obj, load_obj
from PIL import Image
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
import IPython
the_ipython_instance = IPython.get_ipython()
if the_ipython_instance is not None:
the_ipython_instance.magic("load_ext autoreload")
the_ipython_instance.magic("autoreload 2")
the_ipython_instance.magic("matplotlib inline")
['/home/lassepe/miniforge3/envs/latent/lib/python38.zip', '/home/lassepe/miniforge3/envs/latent/lib/python3.8', '/home/lassepe/miniforge3/envs/latent/lib/python3.8/lib-dynload', '', '/home/lassepe/miniforge3/envs/latent/lib/python3.8/site-packages', '/home/lassepe/worktree/latent-safety', '/home/lassepe/worktree/latent-safety/safety_rl', '/home/lassepe/worktree/latent-safety/model_based_irl_torch'] Remove Copy-v0 from registry Remove RepeatCopy-v0 from registry Remove ReversedAddition-v0 from registry Remove ReversedAddition3-v0 from registry Remove DuplicatedInput-v0 from registry Remove Reverse-v0 from registry Remove CartPole-v0 from registry Remove CartPole-v1 from registry Remove MountainCar-v0 from registry Remove MountainCarContinuous-v0 from registry Remove Pendulum-v0 from registry Remove Acrobot-v1 from registry Remove LunarLander-v2 from registry Remove LunarLanderContinuous-v2 from registry Remove BipedalWalker-v3 from registry Remove BipedalWalkerHardcore-v3 from registry Remove CarRacing-v0 from registry Remove Blackjack-v0 from registry Remove KellyCoinflip-v0 from registry Remove KellyCoinflipGeneralized-v0 from registry Remove FrozenLake-v0 from registry Remove FrozenLake8x8-v0 from registry Remove CliffWalking-v0 from registry Remove NChain-v0 from registry Remove Roulette-v0 from registry Remove Taxi-v3 from registry Remove GuessingGame-v0 from registry Remove HotterColder-v0 from registry Remove Reacher-v2 from registry Remove Pusher-v2 from registry Remove Thrower-v2 from registry Remove Striker-v2 from registry Remove InvertedPendulum-v2 from registry Remove InvertedDoublePendulum-v2 from registry Remove HalfCheetah-v2 from registry Remove HalfCheetah-v3 from registry Remove Hopper-v2 from registry Remove Hopper-v3 from registry Remove Swimmer-v2 from registry Remove Swimmer-v3 from registry Remove Walker2d-v2 from registry Remove Walker2d-v3 from registry Remove Ant-v2 from registry Remove Ant-v3 from registry Remove Humanoid-v2 from registry Remove Humanoid-v3 from registry Remove HumanoidStandup-v2 from registry Remove FetchSlide-v1 from registry Remove FetchPickAndPlace-v1 from registry Remove FetchReach-v1 from registry Remove FetchPush-v1 from registry Remove HandReach-v0 from registry Remove HandManipulateBlockRotateZ-v0 from registry Remove HandManipulateBlockRotateZTouchSensors-v0 from registry Remove HandManipulateBlockRotateZTouchSensors-v1 from registry Remove HandManipulateBlockRotateParallel-v0 from registry Remove HandManipulateBlockRotateParallelTouchSensors-v0 from registry Remove HandManipulateBlockRotateParallelTouchSensors-v1 from registry Remove HandManipulateBlockRotateXYZ-v0 from registry Remove HandManipulateBlockRotateXYZTouchSensors-v0 from registry Remove HandManipulateBlockRotateXYZTouchSensors-v1 from registry Remove HandManipulateBlockFull-v0 from registry Remove HandManipulateBlock-v0 from registry Remove HandManipulateBlockTouchSensors-v0 from registry Remove HandManipulateBlockTouchSensors-v1 from registry Remove HandManipulateEggRotate-v0 from registry Remove HandManipulateEggRotateTouchSensors-v0 from registry Remove HandManipulateEggRotateTouchSensors-v1 from registry Remove HandManipulateEggFull-v0 from registry Remove HandManipulateEgg-v0 from registry Remove HandManipulateEggTouchSensors-v0 from registry Remove HandManipulateEggTouchSensors-v1 from registry Remove HandManipulatePenRotate-v0 from registry Remove HandManipulatePenRotateTouchSensors-v0 from registry Remove HandManipulatePenRotateTouchSensors-v1 from registry Remove HandManipulatePenFull-v0 from registry Remove HandManipulatePen-v0 from registry Remove HandManipulatePenTouchSensors-v0 from registry Remove HandManipulatePenTouchSensors-v1 from registry Remove FetchSlideDense-v1 from registry Remove FetchPickAndPlaceDense-v1 from registry Remove FetchReachDense-v1 from registry Remove FetchPushDense-v1 from registry Remove HandReachDense-v0 from registry Remove HandManipulateBlockRotateZDense-v0 from registry Remove HandManipulateBlockRotateZTouchSensorsDense-v0 from registry Remove HandManipulateBlockRotateZTouchSensorsDense-v1 from registry Remove HandManipulateBlockRotateParallelDense-v0 from registry Remove HandManipulateBlockRotateParallelTouchSensorsDense-v0 from registry Remove HandManipulateBlockRotateParallelTouchSensorsDense-v1 from registry Remove HandManipulateBlockRotateXYZDense-v0 from registry Remove HandManipulateBlockRotateXYZTouchSensorsDense-v0 from registry Remove HandManipulateBlockRotateXYZTouchSensorsDense-v1 from registry Remove HandManipulateBlockFullDense-v0 from registry Remove HandManipulateBlockDense-v0 from registry Remove HandManipulateBlockTouchSensorsDense-v0 from registry Remove HandManipulateBlockTouchSensorsDense-v1 from registry Remove HandManipulateEggRotateDense-v0 from registry Remove HandManipulateEggRotateTouchSensorsDense-v0 from registry Remove HandManipulateEggRotateTouchSensorsDense-v1 from registry Remove HandManipulateEggFullDense-v0 from registry Remove HandManipulateEggDense-v0 from registry Remove HandManipulateEggTouchSensorsDense-v0 from registry Remove HandManipulateEggTouchSensorsDense-v1 from registry Remove HandManipulatePenRotateDense-v0 from registry Remove HandManipulatePenRotateTouchSensorsDense-v0 from registry Remove HandManipulatePenRotateTouchSensorsDense-v1 from registry Remove HandManipulatePenFullDense-v0 from registry Remove HandManipulatePenDense-v0 from registry Remove HandManipulatePenTouchSensorsDense-v0 from registry Remove HandManipulatePenTouchSensorsDense-v1 from registry Remove Adventure-v0 from registry Remove Adventure-v4 from registry Remove AdventureDeterministic-v0 from registry Remove AdventureDeterministic-v4 from registry Remove AdventureNoFrameskip-v0 from registry Remove AdventureNoFrameskip-v4 from registry Remove Adventure-ram-v0 from registry Remove Adventure-ram-v4 from registry Remove Adventure-ramDeterministic-v0 from registry Remove Adventure-ramDeterministic-v4 from registry Remove Adventure-ramNoFrameskip-v0 from registry Remove Adventure-ramNoFrameskip-v4 from registry Remove AirRaid-v0 from registry Remove AirRaid-v4 from registry Remove AirRaidDeterministic-v0 from registry Remove AirRaidDeterministic-v4 from registry Remove AirRaidNoFrameskip-v0 from registry Remove AirRaidNoFrameskip-v4 from registry Remove AirRaid-ram-v0 from registry Remove AirRaid-ram-v4 from registry Remove AirRaid-ramDeterministic-v0 from registry Remove AirRaid-ramDeterministic-v4 from registry Remove AirRaid-ramNoFrameskip-v0 from registry Remove AirRaid-ramNoFrameskip-v4 from registry Remove Alien-v0 from registry Remove Alien-v4 from registry Remove AlienDeterministic-v0 from registry Remove AlienDeterministic-v4 from registry Remove AlienNoFrameskip-v0 from registry Remove AlienNoFrameskip-v4 from registry Remove Alien-ram-v0 from registry Remove Alien-ram-v4 from registry Remove Alien-ramDeterministic-v0 from registry Remove Alien-ramDeterministic-v4 from registry Remove Alien-ramNoFrameskip-v0 from registry Remove Alien-ramNoFrameskip-v4 from registry Remove Amidar-v0 from registry Remove Amidar-v4 from registry Remove AmidarDeterministic-v0 from registry Remove AmidarDeterministic-v4 from registry Remove AmidarNoFrameskip-v0 from registry Remove AmidarNoFrameskip-v4 from registry Remove Amidar-ram-v0 from registry Remove Amidar-ram-v4 from registry Remove Amidar-ramDeterministic-v0 from registry Remove Amidar-ramDeterministic-v4 from registry Remove Amidar-ramNoFrameskip-v0 from registry Remove Amidar-ramNoFrameskip-v4 from registry Remove Assault-v0 from registry Remove Assault-v4 from registry Remove AssaultDeterministic-v0 from registry Remove AssaultDeterministic-v4 from registry Remove AssaultNoFrameskip-v0 from registry Remove AssaultNoFrameskip-v4 from registry Remove Assault-ram-v0 from registry Remove Assault-ram-v4 from registry Remove Assault-ramDeterministic-v0 from registry Remove Assault-ramDeterministic-v4 from registry Remove Assault-ramNoFrameskip-v0 from registry Remove Assault-ramNoFrameskip-v4 from registry Remove Asterix-v0 from registry Remove Asterix-v4 from registry Remove AsterixDeterministic-v0 from registry Remove AsterixDeterministic-v4 from registry Remove AsterixNoFrameskip-v0 from registry Remove AsterixNoFrameskip-v4 from registry Remove Asterix-ram-v0 from registry Remove Asterix-ram-v4 from registry Remove Asterix-ramDeterministic-v0 from registry Remove Asterix-ramDeterministic-v4 from registry Remove Asterix-ramNoFrameskip-v0 from registry Remove Asterix-ramNoFrameskip-v4 from registry Remove Asteroids-v0 from registry Remove Asteroids-v4 from registry Remove AsteroidsDeterministic-v0 from registry Remove AsteroidsDeterministic-v4 from registry Remove AsteroidsNoFrameskip-v0 from registry Remove AsteroidsNoFrameskip-v4 from registry Remove Asteroids-ram-v0 from registry Remove Asteroids-ram-v4 from registry Remove Asteroids-ramDeterministic-v0 from registry Remove Asteroids-ramDeterministic-v4 from registry Remove Asteroids-ramNoFrameskip-v0 from registry Remove Asteroids-ramNoFrameskip-v4 from registry Remove Atlantis-v0 from registry Remove Atlantis-v4 from registry Remove AtlantisDeterministic-v0 from registry Remove AtlantisDeterministic-v4 from registry Remove AtlantisNoFrameskip-v0 from registry Remove AtlantisNoFrameskip-v4 from registry Remove Atlantis-ram-v0 from registry Remove Atlantis-ram-v4 from registry Remove Atlantis-ramDeterministic-v0 from registry Remove Atlantis-ramDeterministic-v4 from registry Remove Atlantis-ramNoFrameskip-v0 from registry Remove Atlantis-ramNoFrameskip-v4 from registry Remove BankHeist-v0 from registry Remove BankHeist-v4 from registry Remove BankHeistDeterministic-v0 from registry Remove BankHeistDeterministic-v4 from registry Remove BankHeistNoFrameskip-v0 from registry Remove BankHeistNoFrameskip-v4 from registry Remove BankHeist-ram-v0 from registry Remove BankHeist-ram-v4 from registry Remove BankHeist-ramDeterministic-v0 from registry Remove BankHeist-ramDeterministic-v4 from registry Remove BankHeist-ramNoFrameskip-v0 from registry Remove BankHeist-ramNoFrameskip-v4 from registry Remove BattleZone-v0 from registry Remove BattleZone-v4 from registry Remove BattleZoneDeterministic-v0 from registry Remove BattleZoneDeterministic-v4 from registry Remove BattleZoneNoFrameskip-v0 from registry Remove BattleZoneNoFrameskip-v4 from registry Remove BattleZone-ram-v0 from registry Remove BattleZone-ram-v4 from registry Remove BattleZone-ramDeterministic-v0 from registry Remove BattleZone-ramDeterministic-v4 from registry Remove BattleZone-ramNoFrameskip-v0 from registry Remove BattleZone-ramNoFrameskip-v4 from registry Remove BeamRider-v0 from registry Remove BeamRider-v4 from registry Remove BeamRiderDeterministic-v0 from registry Remove BeamRiderDeterministic-v4 from registry Remove BeamRiderNoFrameskip-v0 from registry Remove BeamRiderNoFrameskip-v4 from registry Remove BeamRider-ram-v0 from registry Remove BeamRider-ram-v4 from registry Remove BeamRider-ramDeterministic-v0 from registry Remove BeamRider-ramDeterministic-v4 from registry Remove BeamRider-ramNoFrameskip-v0 from registry Remove BeamRider-ramNoFrameskip-v4 from registry Remove Berzerk-v0 from registry Remove Berzerk-v4 from registry Remove BerzerkDeterministic-v0 from registry Remove BerzerkDeterministic-v4 from registry Remove BerzerkNoFrameskip-v0 from registry Remove BerzerkNoFrameskip-v4 from registry Remove Berzerk-ram-v0 from registry Remove Berzerk-ram-v4 from registry Remove Berzerk-ramDeterministic-v0 from registry Remove Berzerk-ramDeterministic-v4 from registry Remove Berzerk-ramNoFrameskip-v0 from registry Remove Berzerk-ramNoFrameskip-v4 from registry Remove Bowling-v0 from registry Remove Bowling-v4 from registry Remove BowlingDeterministic-v0 from registry Remove BowlingDeterministic-v4 from registry Remove BowlingNoFrameskip-v0 from registry Remove BowlingNoFrameskip-v4 from registry Remove Bowling-ram-v0 from registry Remove Bowling-ram-v4 from registry Remove Bowling-ramDeterministic-v0 from registry Remove Bowling-ramDeterministic-v4 from registry Remove Bowling-ramNoFrameskip-v0 from registry Remove Bowling-ramNoFrameskip-v4 from registry Remove Boxing-v0 from registry Remove Boxing-v4 from registry Remove BoxingDeterministic-v0 from registry Remove BoxingDeterministic-v4 from registry Remove BoxingNoFrameskip-v0 from registry Remove BoxingNoFrameskip-v4 from registry Remove Boxing-ram-v0 from registry Remove Boxing-ram-v4 from registry Remove Boxing-ramDeterministic-v0 from registry Remove Boxing-ramDeterministic-v4 from registry Remove Boxing-ramNoFrameskip-v0 from registry Remove Boxing-ramNoFrameskip-v4 from registry Remove Breakout-v0 from registry Remove Breakout-v4 from registry Remove BreakoutDeterministic-v0 from registry Remove BreakoutDeterministic-v4 from registry Remove BreakoutNoFrameskip-v0 from registry Remove BreakoutNoFrameskip-v4 from registry Remove Breakout-ram-v0 from registry Remove Breakout-ram-v4 from registry Remove Breakout-ramDeterministic-v0 from registry Remove Breakout-ramDeterministic-v4 from registry Remove Breakout-ramNoFrameskip-v0 from registry Remove Breakout-ramNoFrameskip-v4 from registry Remove Carnival-v0 from registry Remove Carnival-v4 from registry Remove CarnivalDeterministic-v0 from registry Remove CarnivalDeterministic-v4 from registry Remove CarnivalNoFrameskip-v0 from registry Remove CarnivalNoFrameskip-v4 from registry Remove Carnival-ram-v0 from registry Remove Carnival-ram-v4 from registry Remove Carnival-ramDeterministic-v0 from registry Remove Carnival-ramDeterministic-v4 from registry Remove Carnival-ramNoFrameskip-v0 from registry Remove Carnival-ramNoFrameskip-v4 from registry Remove Centipede-v0 from registry Remove Centipede-v4 from registry Remove CentipedeDeterministic-v0 from registry Remove CentipedeDeterministic-v4 from registry Remove CentipedeNoFrameskip-v0 from registry Remove CentipedeNoFrameskip-v4 from registry Remove Centipede-ram-v0 from registry Remove Centipede-ram-v4 from registry Remove Centipede-ramDeterministic-v0 from registry Remove Centipede-ramDeterministic-v4 from registry Remove Centipede-ramNoFrameskip-v0 from registry Remove Centipede-ramNoFrameskip-v4 from registry Remove ChopperCommand-v0 from registry Remove ChopperCommand-v4 from registry Remove ChopperCommandDeterministic-v0 from registry Remove ChopperCommandDeterministic-v4 from registry Remove ChopperCommandNoFrameskip-v0 from registry Remove ChopperCommandNoFrameskip-v4 from registry Remove ChopperCommand-ram-v0 from registry Remove ChopperCommand-ram-v4 from registry Remove ChopperCommand-ramDeterministic-v0 from registry Remove ChopperCommand-ramDeterministic-v4 from registry Remove ChopperCommand-ramNoFrameskip-v0 from registry Remove ChopperCommand-ramNoFrameskip-v4 from registry Remove CrazyClimber-v0 from registry Remove CrazyClimber-v4 from registry Remove CrazyClimberDeterministic-v0 from registry Remove CrazyClimberDeterministic-v4 from registry Remove CrazyClimberNoFrameskip-v0 from registry Remove CrazyClimberNoFrameskip-v4 from registry Remove CrazyClimber-ram-v0 from registry Remove CrazyClimber-ram-v4 from registry Remove CrazyClimber-ramDeterministic-v0 from registry Remove CrazyClimber-ramDeterministic-v4 from registry Remove CrazyClimber-ramNoFrameskip-v0 from registry Remove CrazyClimber-ramNoFrameskip-v4 from registry Remove Defender-v0 from registry Remove Defender-v4 from registry Remove DefenderDeterministic-v0 from registry Remove DefenderDeterministic-v4 from registry Remove DefenderNoFrameskip-v0 from registry Remove DefenderNoFrameskip-v4 from registry Remove Defender-ram-v0 from registry Remove Defender-ram-v4 from registry Remove Defender-ramDeterministic-v0 from registry Remove Defender-ramDeterministic-v4 from registry Remove Defender-ramNoFrameskip-v0 from registry Remove Defender-ramNoFrameskip-v4 from registry Remove DemonAttack-v0 from registry Remove DemonAttack-v4 from registry Remove DemonAttackDeterministic-v0 from registry Remove DemonAttackDeterministic-v4 from registry Remove DemonAttackNoFrameskip-v0 from registry Remove DemonAttackNoFrameskip-v4 from registry Remove DemonAttack-ram-v0 from registry Remove DemonAttack-ram-v4 from registry Remove DemonAttack-ramDeterministic-v0 from registry Remove DemonAttack-ramDeterministic-v4 from registry Remove DemonAttack-ramNoFrameskip-v0 from registry Remove DemonAttack-ramNoFrameskip-v4 from registry Remove DoubleDunk-v0 from registry Remove DoubleDunk-v4 from registry Remove DoubleDunkDeterministic-v0 from registry Remove DoubleDunkDeterministic-v4 from registry Remove DoubleDunkNoFrameskip-v0 from registry Remove DoubleDunkNoFrameskip-v4 from registry Remove DoubleDunk-ram-v0 from registry Remove DoubleDunk-ram-v4 from registry Remove DoubleDunk-ramDeterministic-v0 from registry Remove DoubleDunk-ramDeterministic-v4 from registry Remove DoubleDunk-ramNoFrameskip-v0 from registry Remove DoubleDunk-ramNoFrameskip-v4 from registry Remove ElevatorAction-v0 from registry Remove ElevatorAction-v4 from registry Remove ElevatorActionDeterministic-v0 from registry Remove ElevatorActionDeterministic-v4 from registry Remove ElevatorActionNoFrameskip-v0 from registry Remove ElevatorActionNoFrameskip-v4 from registry Remove ElevatorAction-ram-v0 from registry Remove ElevatorAction-ram-v4 from registry Remove ElevatorAction-ramDeterministic-v0 from registry Remove ElevatorAction-ramDeterministic-v4 from registry Remove ElevatorAction-ramNoFrameskip-v0 from registry Remove ElevatorAction-ramNoFrameskip-v4 from registry Remove Enduro-v0 from registry Remove Enduro-v4 from registry Remove EnduroDeterministic-v0 from registry Remove EnduroDeterministic-v4 from registry Remove EnduroNoFrameskip-v0 from registry Remove EnduroNoFrameskip-v4 from registry Remove Enduro-ram-v0 from registry Remove Enduro-ram-v4 from registry Remove Enduro-ramDeterministic-v0 from registry Remove Enduro-ramDeterministic-v4 from registry Remove Enduro-ramNoFrameskip-v0 from registry Remove Enduro-ramNoFrameskip-v4 from registry Remove FishingDerby-v0 from registry Remove FishingDerby-v4 from registry Remove FishingDerbyDeterministic-v0 from registry Remove FishingDerbyDeterministic-v4 from registry Remove FishingDerbyNoFrameskip-v0 from registry Remove FishingDerbyNoFrameskip-v4 from registry Remove FishingDerby-ram-v0 from registry Remove FishingDerby-ram-v4 from registry Remove FishingDerby-ramDeterministic-v0 from registry Remove FishingDerby-ramDeterministic-v4 from registry Remove FishingDerby-ramNoFrameskip-v0 from registry Remove FishingDerby-ramNoFrameskip-v4 from registry Remove Freeway-v0 from registry Remove Freeway-v4 from registry Remove FreewayDeterministic-v0 from registry Remove FreewayDeterministic-v4 from registry Remove FreewayNoFrameskip-v0 from registry Remove FreewayNoFrameskip-v4 from registry Remove Freeway-ram-v0 from registry Remove Freeway-ram-v4 from registry Remove Freeway-ramDeterministic-v0 from registry Remove Freeway-ramDeterministic-v4 from registry Remove Freeway-ramNoFrameskip-v0 from registry Remove Freeway-ramNoFrameskip-v4 from registry Remove Frostbite-v0 from registry Remove Frostbite-v4 from registry Remove FrostbiteDeterministic-v0 from registry Remove FrostbiteDeterministic-v4 from registry Remove FrostbiteNoFrameskip-v0 from registry Remove FrostbiteNoFrameskip-v4 from registry Remove Frostbite-ram-v0 from registry Remove Frostbite-ram-v4 from registry Remove Frostbite-ramDeterministic-v0 from registry Remove Frostbite-ramDeterministic-v4 from registry Remove Frostbite-ramNoFrameskip-v0 from registry Remove Frostbite-ramNoFrameskip-v4 from registry Remove Gopher-v0 from registry Remove Gopher-v4 from registry Remove GopherDeterministic-v0 from registry Remove GopherDeterministic-v4 from registry Remove GopherNoFrameskip-v0 from registry Remove GopherNoFrameskip-v4 from registry Remove Gopher-ram-v0 from registry Remove Gopher-ram-v4 from registry Remove Gopher-ramDeterministic-v0 from registry Remove Gopher-ramDeterministic-v4 from registry Remove Gopher-ramNoFrameskip-v0 from registry Remove Gopher-ramNoFrameskip-v4 from registry Remove Gravitar-v0 from registry Remove Gravitar-v4 from registry Remove GravitarDeterministic-v0 from registry Remove GravitarDeterministic-v4 from registry Remove GravitarNoFrameskip-v0 from registry Remove GravitarNoFrameskip-v4 from registry Remove Gravitar-ram-v0 from registry Remove Gravitar-ram-v4 from registry Remove Gravitar-ramDeterministic-v0 from registry Remove Gravitar-ramDeterministic-v4 from registry Remove Gravitar-ramNoFrameskip-v0 from registry Remove Gravitar-ramNoFrameskip-v4 from registry Remove Hero-v0 from registry Remove Hero-v4 from registry Remove HeroDeterministic-v0 from registry Remove HeroDeterministic-v4 from registry Remove HeroNoFrameskip-v0 from registry Remove HeroNoFrameskip-v4 from registry Remove Hero-ram-v0 from registry Remove Hero-ram-v4 from registry Remove Hero-ramDeterministic-v0 from registry Remove Hero-ramDeterministic-v4 from registry Remove Hero-ramNoFrameskip-v0 from registry Remove Hero-ramNoFrameskip-v4 from registry Remove IceHockey-v0 from registry Remove IceHockey-v4 from registry Remove IceHockeyDeterministic-v0 from registry Remove IceHockeyDeterministic-v4 from registry Remove IceHockeyNoFrameskip-v0 from registry Remove IceHockeyNoFrameskip-v4 from registry Remove IceHockey-ram-v0 from registry Remove IceHockey-ram-v4 from registry Remove IceHockey-ramDeterministic-v0 from registry Remove IceHockey-ramDeterministic-v4 from registry Remove IceHockey-ramNoFrameskip-v0 from registry Remove IceHockey-ramNoFrameskip-v4 from registry Remove Jamesbond-v0 from registry Remove Jamesbond-v4 from registry Remove JamesbondDeterministic-v0 from registry Remove JamesbondDeterministic-v4 from registry Remove JamesbondNoFrameskip-v0 from registry Remove JamesbondNoFrameskip-v4 from registry Remove Jamesbond-ram-v0 from registry Remove Jamesbond-ram-v4 from registry Remove Jamesbond-ramDeterministic-v0 from registry Remove Jamesbond-ramDeterministic-v4 from registry Remove Jamesbond-ramNoFrameskip-v0 from registry Remove Jamesbond-ramNoFrameskip-v4 from registry Remove JourneyEscape-v0 from registry Remove JourneyEscape-v4 from registry Remove JourneyEscapeDeterministic-v0 from registry Remove JourneyEscapeDeterministic-v4 from registry Remove JourneyEscapeNoFrameskip-v0 from registry Remove JourneyEscapeNoFrameskip-v4 from registry Remove JourneyEscape-ram-v0 from registry Remove JourneyEscape-ram-v4 from registry Remove JourneyEscape-ramDeterministic-v0 from registry Remove JourneyEscape-ramDeterministic-v4 from registry Remove JourneyEscape-ramNoFrameskip-v0 from registry Remove JourneyEscape-ramNoFrameskip-v4 from registry Remove Kangaroo-v0 from registry Remove Kangaroo-v4 from registry Remove KangarooDeterministic-v0 from registry Remove KangarooDeterministic-v4 from registry Remove KangarooNoFrameskip-v0 from registry Remove KangarooNoFrameskip-v4 from registry Remove Kangaroo-ram-v0 from registry Remove Kangaroo-ram-v4 from registry Remove Kangaroo-ramDeterministic-v0 from registry Remove Kangaroo-ramDeterministic-v4 from registry Remove Kangaroo-ramNoFrameskip-v0 from registry Remove Kangaroo-ramNoFrameskip-v4 from registry Remove Krull-v0 from registry Remove Krull-v4 from registry Remove KrullDeterministic-v0 from registry Remove KrullDeterministic-v4 from registry Remove KrullNoFrameskip-v0 from registry Remove KrullNoFrameskip-v4 from registry Remove Krull-ram-v0 from registry Remove Krull-ram-v4 from registry Remove Krull-ramDeterministic-v0 from registry Remove Krull-ramDeterministic-v4 from registry Remove Krull-ramNoFrameskip-v0 from registry Remove Krull-ramNoFrameskip-v4 from registry Remove KungFuMaster-v0 from registry Remove KungFuMaster-v4 from registry Remove KungFuMasterDeterministic-v0 from registry Remove KungFuMasterDeterministic-v4 from registry Remove KungFuMasterNoFrameskip-v0 from registry Remove KungFuMasterNoFrameskip-v4 from registry Remove KungFuMaster-ram-v0 from registry Remove KungFuMaster-ram-v4 from registry Remove KungFuMaster-ramDeterministic-v0 from registry Remove KungFuMaster-ramDeterministic-v4 from registry Remove KungFuMaster-ramNoFrameskip-v0 from registry Remove KungFuMaster-ramNoFrameskip-v4 from registry Remove MontezumaRevenge-v0 from registry Remove MontezumaRevenge-v4 from registry Remove MontezumaRevengeDeterministic-v0 from registry Remove MontezumaRevengeDeterministic-v4 from registry Remove MontezumaRevengeNoFrameskip-v0 from registry Remove MontezumaRevengeNoFrameskip-v4 from registry Remove MontezumaRevenge-ram-v0 from registry Remove MontezumaRevenge-ram-v4 from registry Remove MontezumaRevenge-ramDeterministic-v0 from registry Remove MontezumaRevenge-ramDeterministic-v4 from registry Remove MontezumaRevenge-ramNoFrameskip-v0 from registry Remove MontezumaRevenge-ramNoFrameskip-v4 from registry Remove MsPacman-v0 from registry Remove MsPacman-v4 from registry Remove MsPacmanDeterministic-v0 from registry Remove MsPacmanDeterministic-v4 from registry Remove MsPacmanNoFrameskip-v0 from registry Remove MsPacmanNoFrameskip-v4 from registry Remove MsPacman-ram-v0 from registry Remove MsPacman-ram-v4 from registry Remove MsPacman-ramDeterministic-v0 from registry Remove MsPacman-ramDeterministic-v4 from registry Remove MsPacman-ramNoFrameskip-v0 from registry Remove MsPacman-ramNoFrameskip-v4 from registry Remove NameThisGame-v0 from registry Remove NameThisGame-v4 from registry Remove NameThisGameDeterministic-v0 from registry Remove NameThisGameDeterministic-v4 from registry Remove NameThisGameNoFrameskip-v0 from registry Remove NameThisGameNoFrameskip-v4 from registry Remove NameThisGame-ram-v0 from registry Remove NameThisGame-ram-v4 from registry Remove NameThisGame-ramDeterministic-v0 from registry Remove NameThisGame-ramDeterministic-v4 from registry Remove NameThisGame-ramNoFrameskip-v0 from registry Remove NameThisGame-ramNoFrameskip-v4 from registry Remove Phoenix-v0 from registry Remove Phoenix-v4 from registry Remove PhoenixDeterministic-v0 from registry Remove PhoenixDeterministic-v4 from registry Remove PhoenixNoFrameskip-v0 from registry Remove PhoenixNoFrameskip-v4 from registry Remove Phoenix-ram-v0 from registry Remove Phoenix-ram-v4 from registry Remove Phoenix-ramDeterministic-v0 from registry Remove Phoenix-ramDeterministic-v4 from registry Remove Phoenix-ramNoFrameskip-v0 from registry Remove Phoenix-ramNoFrameskip-v4 from registry Remove Pitfall-v0 from registry Remove Pitfall-v4 from registry Remove PitfallDeterministic-v0 from registry Remove PitfallDeterministic-v4 from registry Remove PitfallNoFrameskip-v0 from registry Remove PitfallNoFrameskip-v4 from registry Remove Pitfall-ram-v0 from registry Remove Pitfall-ram-v4 from registry Remove Pitfall-ramDeterministic-v0 from registry Remove Pitfall-ramDeterministic-v4 from registry Remove Pitfall-ramNoFrameskip-v0 from registry Remove Pitfall-ramNoFrameskip-v4 from registry Remove Pong-v0 from registry Remove Pong-v4 from registry Remove PongDeterministic-v0 from registry Remove PongDeterministic-v4 from registry Remove PongNoFrameskip-v0 from registry Remove PongNoFrameskip-v4 from registry Remove Pong-ram-v0 from registry Remove Pong-ram-v4 from registry Remove Pong-ramDeterministic-v0 from registry Remove Pong-ramDeterministic-v4 from registry Remove Pong-ramNoFrameskip-v0 from registry Remove Pong-ramNoFrameskip-v4 from registry Remove Pooyan-v0 from registry Remove Pooyan-v4 from registry Remove PooyanDeterministic-v0 from registry Remove PooyanDeterministic-v4 from registry Remove PooyanNoFrameskip-v0 from registry Remove PooyanNoFrameskip-v4 from registry Remove Pooyan-ram-v0 from registry Remove Pooyan-ram-v4 from registry Remove Pooyan-ramDeterministic-v0 from registry Remove Pooyan-ramDeterministic-v4 from registry Remove Pooyan-ramNoFrameskip-v0 from registry Remove Pooyan-ramNoFrameskip-v4 from registry Remove PrivateEye-v0 from registry Remove PrivateEye-v4 from registry Remove PrivateEyeDeterministic-v0 from registry Remove PrivateEyeDeterministic-v4 from registry Remove PrivateEyeNoFrameskip-v0 from registry Remove PrivateEyeNoFrameskip-v4 from registry Remove PrivateEye-ram-v0 from registry Remove PrivateEye-ram-v4 from registry Remove PrivateEye-ramDeterministic-v0 from registry Remove PrivateEye-ramDeterministic-v4 from registry Remove PrivateEye-ramNoFrameskip-v0 from registry Remove PrivateEye-ramNoFrameskip-v4 from registry Remove Qbert-v0 from registry Remove Qbert-v4 from registry Remove QbertDeterministic-v0 from registry Remove QbertDeterministic-v4 from registry Remove QbertNoFrameskip-v0 from registry Remove QbertNoFrameskip-v4 from registry Remove Qbert-ram-v0 from registry Remove Qbert-ram-v4 from registry Remove Qbert-ramDeterministic-v0 from registry Remove Qbert-ramDeterministic-v4 from registry Remove Qbert-ramNoFrameskip-v0 from registry Remove Qbert-ramNoFrameskip-v4 from registry Remove Riverraid-v0 from registry Remove Riverraid-v4 from registry Remove RiverraidDeterministic-v0 from registry Remove RiverraidDeterministic-v4 from registry Remove RiverraidNoFrameskip-v0 from registry Remove RiverraidNoFrameskip-v4 from registry Remove Riverraid-ram-v0 from registry Remove Riverraid-ram-v4 from registry Remove Riverraid-ramDeterministic-v0 from registry Remove Riverraid-ramDeterministic-v4 from registry Remove Riverraid-ramNoFrameskip-v0 from registry Remove Riverraid-ramNoFrameskip-v4 from registry Remove RoadRunner-v0 from registry Remove RoadRunner-v4 from registry Remove RoadRunnerDeterministic-v0 from registry Remove RoadRunnerDeterministic-v4 from registry Remove RoadRunnerNoFrameskip-v0 from registry Remove RoadRunnerNoFrameskip-v4 from registry Remove RoadRunner-ram-v0 from registry Remove RoadRunner-ram-v4 from registry Remove RoadRunner-ramDeterministic-v0 from registry Remove RoadRunner-ramDeterministic-v4 from registry Remove RoadRunner-ramNoFrameskip-v0 from registry Remove RoadRunner-ramNoFrameskip-v4 from registry Remove Robotank-v0 from registry Remove Robotank-v4 from registry Remove RobotankDeterministic-v0 from registry Remove RobotankDeterministic-v4 from registry Remove RobotankNoFrameskip-v0 from registry Remove RobotankNoFrameskip-v4 from registry Remove Robotank-ram-v0 from registry Remove Robotank-ram-v4 from registry Remove Robotank-ramDeterministic-v0 from registry Remove Robotank-ramDeterministic-v4 from registry Remove Robotank-ramNoFrameskip-v0 from registry Remove Robotank-ramNoFrameskip-v4 from registry Remove Seaquest-v0 from registry Remove Seaquest-v4 from registry Remove SeaquestDeterministic-v0 from registry Remove SeaquestDeterministic-v4 from registry Remove SeaquestNoFrameskip-v0 from registry Remove SeaquestNoFrameskip-v4 from registry Remove Seaquest-ram-v0 from registry Remove Seaquest-ram-v4 from registry Remove Seaquest-ramDeterministic-v0 from registry Remove Seaquest-ramDeterministic-v4 from registry Remove Seaquest-ramNoFrameskip-v0 from registry Remove Seaquest-ramNoFrameskip-v4 from registry Remove Skiing-v0 from registry Remove Skiing-v4 from registry Remove SkiingDeterministic-v0 from registry Remove SkiingDeterministic-v4 from registry Remove SkiingNoFrameskip-v0 from registry Remove SkiingNoFrameskip-v4 from registry Remove Skiing-ram-v0 from registry Remove Skiing-ram-v4 from registry Remove Skiing-ramDeterministic-v0 from registry Remove Skiing-ramDeterministic-v4 from registry Remove Skiing-ramNoFrameskip-v0 from registry Remove Skiing-ramNoFrameskip-v4 from registry Remove Solaris-v0 from registry Remove Solaris-v4 from registry Remove SolarisDeterministic-v0 from registry Remove SolarisDeterministic-v4 from registry Remove SolarisNoFrameskip-v0 from registry Remove SolarisNoFrameskip-v4 from registry Remove Solaris-ram-v0 from registry Remove Solaris-ram-v4 from registry Remove Solaris-ramDeterministic-v0 from registry Remove Solaris-ramDeterministic-v4 from registry Remove Solaris-ramNoFrameskip-v0 from registry Remove Solaris-ramNoFrameskip-v4 from registry Remove SpaceInvaders-v0 from registry Remove SpaceInvaders-v4 from registry Remove SpaceInvadersDeterministic-v0 from registry Remove SpaceInvadersDeterministic-v4 from registry Remove SpaceInvadersNoFrameskip-v0 from registry Remove SpaceInvadersNoFrameskip-v4 from registry Remove SpaceInvaders-ram-v0 from registry Remove SpaceInvaders-ram-v4 from registry Remove SpaceInvaders-ramDeterministic-v0 from registry Remove SpaceInvaders-ramDeterministic-v4 from registry Remove SpaceInvaders-ramNoFrameskip-v0 from registry Remove SpaceInvaders-ramNoFrameskip-v4 from registry Remove StarGunner-v0 from registry Remove StarGunner-v4 from registry Remove StarGunnerDeterministic-v0 from registry Remove StarGunnerDeterministic-v4 from registry Remove StarGunnerNoFrameskip-v0 from registry Remove StarGunnerNoFrameskip-v4 from registry Remove StarGunner-ram-v0 from registry Remove StarGunner-ram-v4 from registry Remove StarGunner-ramDeterministic-v0 from registry Remove StarGunner-ramDeterministic-v4 from registry Remove StarGunner-ramNoFrameskip-v0 from registry Remove StarGunner-ramNoFrameskip-v4 from registry Remove Tennis-v0 from registry Remove Tennis-v4 from registry Remove TennisDeterministic-v0 from registry Remove TennisDeterministic-v4 from registry Remove TennisNoFrameskip-v0 from registry Remove TennisNoFrameskip-v4 from registry Remove Tennis-ram-v0 from registry Remove Tennis-ram-v4 from registry Remove Tennis-ramDeterministic-v0 from registry Remove Tennis-ramDeterministic-v4 from registry Remove Tennis-ramNoFrameskip-v0 from registry Remove Tennis-ramNoFrameskip-v4 from registry Remove TimePilot-v0 from registry Remove TimePilot-v4 from registry Remove TimePilotDeterministic-v0 from registry Remove TimePilotDeterministic-v4 from registry Remove TimePilotNoFrameskip-v0 from registry Remove TimePilotNoFrameskip-v4 from registry Remove TimePilot-ram-v0 from registry Remove TimePilot-ram-v4 from registry Remove TimePilot-ramDeterministic-v0 from registry Remove TimePilot-ramDeterministic-v4 from registry Remove TimePilot-ramNoFrameskip-v0 from registry Remove TimePilot-ramNoFrameskip-v4 from registry Remove Tutankham-v0 from registry Remove Tutankham-v4 from registry Remove TutankhamDeterministic-v0 from registry Remove TutankhamDeterministic-v4 from registry Remove TutankhamNoFrameskip-v0 from registry Remove TutankhamNoFrameskip-v4 from registry Remove Tutankham-ram-v0 from registry Remove Tutankham-ram-v4 from registry Remove Tutankham-ramDeterministic-v0 from registry Remove Tutankham-ramDeterministic-v4 from registry Remove Tutankham-ramNoFrameskip-v0 from registry Remove Tutankham-ramNoFrameskip-v4 from registry Remove UpNDown-v0 from registry Remove UpNDown-v4 from registry Remove UpNDownDeterministic-v0 from registry Remove UpNDownDeterministic-v4 from registry Remove UpNDownNoFrameskip-v0 from registry Remove UpNDownNoFrameskip-v4 from registry Remove UpNDown-ram-v0 from registry Remove UpNDown-ram-v4 from registry Remove UpNDown-ramDeterministic-v0 from registry Remove UpNDown-ramDeterministic-v4 from registry Remove UpNDown-ramNoFrameskip-v0 from registry Remove UpNDown-ramNoFrameskip-v4 from registry Remove Venture-v0 from registry Remove Venture-v4 from registry Remove VentureDeterministic-v0 from registry Remove VentureDeterministic-v4 from registry Remove VentureNoFrameskip-v0 from registry Remove VentureNoFrameskip-v4 from registry Remove Venture-ram-v0 from registry Remove Venture-ram-v4 from registry Remove Venture-ramDeterministic-v0 from registry Remove Venture-ramDeterministic-v4 from registry Remove Venture-ramNoFrameskip-v0 from registry Remove Venture-ramNoFrameskip-v4 from registry Remove VideoPinball-v0 from registry Remove VideoPinball-v4 from registry Remove VideoPinballDeterministic-v0 from registry Remove VideoPinballDeterministic-v4 from registry Remove VideoPinballNoFrameskip-v0 from registry Remove VideoPinballNoFrameskip-v4 from registry Remove VideoPinball-ram-v0 from registry Remove VideoPinball-ram-v4 from registry Remove VideoPinball-ramDeterministic-v0 from registry Remove VideoPinball-ramDeterministic-v4 from registry Remove VideoPinball-ramNoFrameskip-v0 from registry Remove VideoPinball-ramNoFrameskip-v4 from registry Remove WizardOfWor-v0 from registry Remove WizardOfWor-v4 from registry Remove WizardOfWorDeterministic-v0 from registry Remove WizardOfWorDeterministic-v4 from registry Remove WizardOfWorNoFrameskip-v0 from registry Remove WizardOfWorNoFrameskip-v4 from registry Remove WizardOfWor-ram-v0 from registry Remove WizardOfWor-ram-v4 from registry Remove WizardOfWor-ramDeterministic-v0 from registry Remove WizardOfWor-ramDeterministic-v4 from registry Remove WizardOfWor-ramNoFrameskip-v0 from registry Remove WizardOfWor-ramNoFrameskip-v4 from registry Remove YarsRevenge-v0 from registry Remove YarsRevenge-v4 from registry Remove YarsRevengeDeterministic-v0 from registry Remove YarsRevengeDeterministic-v4 from registry Remove YarsRevengeNoFrameskip-v0 from registry Remove YarsRevengeNoFrameskip-v4 from registry Remove YarsRevenge-ram-v0 from registry Remove YarsRevenge-ram-v4 from registry Remove YarsRevenge-ramDeterministic-v0 from registry Remove YarsRevenge-ramDeterministic-v4 from registry Remove YarsRevenge-ramNoFrameskip-v0 from registry Remove YarsRevenge-ramNoFrameskip-v4 from registry Remove Zaxxon-v0 from registry Remove Zaxxon-v4 from registry Remove ZaxxonDeterministic-v0 from registry Remove ZaxxonDeterministic-v4 from registry Remove ZaxxonNoFrameskip-v0 from registry Remove ZaxxonNoFrameskip-v4 from registry Remove Zaxxon-ram-v0 from registry Remove Zaxxon-ram-v4 from registry Remove Zaxxon-ramDeterministic-v0 from registry Remove Zaxxon-ramDeterministic-v4 from registry Remove Zaxxon-ramNoFrameskip-v0 from registry Remove Zaxxon-ramNoFrameskip-v4 from registry Remove CubeCrash-v0 from registry Remove CubeCrashSparse-v0 from registry Remove CubeCrashScreenBecomesBlack-v0 from registry Remove MemorizeDigits-v0 from registry
<ipython-input-1-eac31f6e0c43>:21: DeprecationWarning: `magic(...)` is deprecated since IPython 0.13 (warning added in 8.1), use run_line_magic(magic_name, parameter_s).
the_ipython_instance.magic("load_ext autoreload")
<ipython-input-1-eac31f6e0c43>:22: DeprecationWarning: `magic(...)` is deprecated since IPython 0.13 (warning added in 8.1), use run_line_magic(magic_name, parameter_s).
the_ipython_instance.magic("autoreload 2")
<ipython-input-1-eac31f6e0c43>:23: DeprecationWarning: `magic(...)` is deprecated since IPython 0.13 (warning added in 8.1), use run_line_magic(magic_name, parameter_s).
the_ipython_instance.magic("matplotlib inline")
In [ ]:
def load_best_agent(config, environment_info):
agent = RARL_wm.construct_agent(config, environment_info)
# TODO get this index from the training dict based on metrics rather than hard-coding
restore_idx = 150_000
agent.restore(
restore_idx, os.path.join(project_root, environment_info["outFolder"])
)
return agent
def compute_value_funtion_metrics(env, ground_truth_brt, q_func):
nx = ground_truth_brt.shape[0]
ny = ground_truth_brt.shape[1]
# sub-sample theta indices to reduce computation
theta_indices = np.linspace(0, ground_truth_brt.shape[2] - 1, 3, dtype=int)
slices = []
for theta_idx in theta_indices:
# map index back to angle
theta = theta_idx * 2 * np.pi / ground_truth_brt.shape[2]
slice = env.get_value(q_func, theta=theta, nx=nx, ny=ny)
slices.append(slice)
v_nn = np.stack(slices, axis=2)
v_grid = ground_truth_brt[:, :, theta_indices]
tn, tp, fn, fp = env.confusion(v_nn, v_grid)
accuracy = (tp + tn) / (tp + tn + fp + fn)
if tp + fp < 1e-6:
precision = 0
else:
precision = tp / (tp + fp)
if tp + fn < 1e-6:
recall = 0
else:
recall = tp / (tp + fn)
if precision + recall < 1e-6:
f1 = 0
else:
f1 = 2 * precision * recall / (precision + recall)
metrics = {
"tn": tn,
"tp": tp,
"fn": fn,
"fp": fp,
"accuracy": accuracy,
"precision": precision,
"recall": recall,
"f1": f1,
}
return metrics
def get_grid_value_for_state(env, grid, x, y, theta):
"""
Find the neirest value in the grid for the given state.
"""
nx = grid.shape[0]
ny = grid.shape[1]
ntheta = grid.shape[2]
dx = (env.bounds[0, 1] - env.bounds[0, 0]) / nx
dy = (env.bounds[1, 1] - env.bounds[1, 0]) / ny
dtheta = 2 * np.pi / grid.shape[2]
x_idx = min(int((x - env.bounds[0, 0]) / dx), nx - 1)
y_idx = min(int((y - env.bounds[1, 0]) / dy), ny - 1)
# for theta, make sure to wrap the angle to 0-2pi
theta_idx = min(int((theta % (2 * np.pi)) / dtheta), ntheta - 1)
return grid[x_idx, y_idx, theta_idx]
def collect_rollout_data(
env,
agent,
position_gridsize,
angle_gridsize,
output_folder,
output_prefix,
enable_observation_feedback=True,
):
rollout_data = {}
thetas = np.linspace(0, 2 * math.pi, angle_gridsize, endpoint=False)
os.makedirs(output_folder, exist_ok=True)
for theta_idx, theta in enumerate(thetas):
theta_deg = math.degrees(theta)
print(
f"Collecting rollouts for theta = {theta_deg:.0f} degrees; {theta_idx+1}/{angle_gridsize}"
)
_, _, infos = env.plot_trajectories(
q_func=agent.Q_network,
num_rnd_traj=10,
theta=theta,
# convert to degrees for filename
save_dir=os.path.join(
output_folder,
output_prefix + f"safe_rollouts_theta{theta_deg:.0f}deg.png",
),
return_infos=True,
enable_observation_feedback=enable_observation_feedback,
wait_for_all_metrics_to_predict_failure=True,
position_gridsize=position_gridsize,
)
rollout_data[theta_idx] = infos
return rollout_data
def evaluate_rollout_data(env, rollout_data, ground_truth_brt):
evaluated_rollout_data = []
for theta_data in rollout_data.values():
for rollout_info in theta_data.values():
# extract relevant information from the rollout
# ground truth metrics
ground_truth_metrics = rollout_info["groundtruth_metrics"]
ground_truth_initial_state = ground_truth_metrics["traj"][0]
ground_truth_initial_value = get_grid_value_for_state(
env, ground_truth_brt, *ground_truth_initial_state
)
ground_truth_failure_margin = ground_truth_metrics["minV"]
ground_truth_failure_time = next(
(
idx
for idx, value in enumerate(ground_truth_metrics["valueList"])
if value < 0
),
None,
)
# learned metrics
learned_metrics = rollout_info["learned_metrics"]
learned_safety_time = next(
(
idx
for idx, value in enumerate(learned_metrics["actionValueList"])
if value < 0
),
None,
)
# high-level classification of the rollout according to main features
is_feasible = ground_truth_initial_value >= 0
is_safe = ground_truth_failure_margin >= 0
assert is_safe == (ground_truth_failure_time is None)
is_learning_classification_correct = (
is_safe and (learned_safety_time is None)
) or (
(not is_safe)
and (learned_safety_time is not None)
and (learned_safety_time <= ground_truth_failure_time)
)
# aggregate all the data
evaluated_rollout_data.append(
{
"ground_truth_initial_value": ground_truth_initial_value,
"ground_truth_failure_time": ground_truth_failure_time,
"learned_failure_time": learned_safety_time,
"is_feasible": is_feasible,
"is_safe": is_safe,
"is_learning_classification_correct": is_learning_classification_correct,
"learned_metrics": learned_metrics,
}
)
return pd.DataFrame(evaluated_rollout_data)
def generate_representative_rollout_videos(env, evaluated_rollouts, output_folder, output_prefix):
# find up to 5 safe and 5 unsafe rollouts and generate a video for each of them
safe_rollouts = evaluated_rollouts[evaluated_rollouts["is_safe"]].head(5)
for idx, row in safe_rollouts.iterrows():
print(f"Generating video for safe rollout {idx}")
rollout_data = row["learned_metrics"]
decode_video_from_feature_sequence(env, rollout_data, os.path.join(output_folder, f"{output_prefix}_safe_{idx}.mp4"))
unsafe_rollouts = evaluated_rollouts[~evaluated_rollouts["is_safe"]].head(5)
for idx, row in unsafe_rollouts.iterrows():
print(f"Generating video for unsafe rollout {idx}")
rollout_data = row["learned_metrics"]
decode_video_from_feature_sequence(env, rollout_data, os.path.join(output_folder, f"{output_prefix}_unsafe_{idx}.mp4"))
def video_from_array(image_sequence_array, output_filename = "output_video.mp4"):
import cv2
# Assuming `image_sequence` is your tensor with shape (time, height, width, 3)
# Normalize the image values to [0, 255] if they are not already in this range
image_sequence_array = (image_sequence_array * 255).clip(0, 255).astype(np.uint8)
# Define video properties
height, width, _ = image_sequence_array.shape[1:]
fps = 30 # Set frames per second
# Create a VideoWriter object
fourcc = cv2.VideoWriter_fourcc(*"mp4v") # Codec for MP4
video_writer = cv2.VideoWriter(output_filename, fourcc, fps, (width, height))
# Write frames to the video
for frame in image_sequence_array:
video_writer.write(frame)
# Release the VideoWriter
video_writer.release()
print(f"Video saved as {output_filename}")
return output_filename
def decode_video_from_feature_sequence(env, rollout, output_filename):
# feature_sequence_batch = torch.FloatTensor(d["learned_metrics"]["traj"]).to(env.device).squeeze(1)
# layout: (batch, time, features)
feature_sequence_batch = torch.FloatTensor(rollout["traj"]).to(env.device).unsqueeze(0).squeeze(2)
image_sequence = env.car.wm.heads["decoder"](feature_sequence_batch)["image"].mode().squeeze(0).cpu().detach().numpy()
# convert tensor of (time, npx, npy, 3) to mp4 video
video_from_array(image_sequence, output_filename)
def visualize_evaluated_rollout_stats(evaluated_rollouts, title):
# create a sunburst chart of the results
# https://plotly.com/python/sunburst-charts/
import plotly.express as px
# Sample dataframe based on the provided layout
df = evaluated_rollouts
# Define labels for each level in the sunburst chart
df["feasibility"] = df["is_feasible"].map({True: "feasible", False: "infeasible"})
df["safety"] = df["is_safe"].map({True: "safe", False: "unsafe"})
df["classification"] = df["is_learning_classification_correct"].map(
{True: "true class.", False: "false class."}
)
# Generate the sunburst plot
fig = px.sunburst(
df,
path=["feasibility", "safety", "classification"],
title=title,
color="safety",
color_discrete_map={
"safe": "lightgreen",
"unsafe": "orange",
"(?)": "lightblue",
},
)
fig.update_traces(textinfo="label+value")
# set root color to white
return fig
def evaluate(
env,
agent,
ground_truth_brt,
experiment_name,
position_gridsize,
angle_gridsize,
reproduce_value_function=True,
reproduce_closed_loop_rollouts=True,
reproduce_open_loop_rollouts=True,
generate_videos=True,
show_plots=True,
):
output_folder = os.path.join(
project_root, default_environment_info["outFolder"], experiment_name
)
os.makedirs(output_folder, exist_ok=True)
# ------------------------------------------- value function
value_function_metrics_path = os.path.join(
output_folder, experiment_name + "_value_function_metrics"
)
if reproduce_value_function or not os.path.exists(
f"{value_function_metrics_path}.pkl"
):
value_function_metrics = compute_value_funtion_metrics(
env, ground_truth_brt, agent.Q_network
)
save_obj(value_function_metrics, value_function_metrics_path)
value_function_metrics = load_obj(value_function_metrics_path)
# pretty print the metrics
for key, value in value_function_metrics.items():
print(f"{key}: {value:.3f}")
# ----------------------- closed-loop rollout data collection
closed_loop_rollout_data_path = os.path.join(
output_folder, experiment_name + "closed_loop_rollout_data"
)
if reproduce_closed_loop_rollouts or not os.path.exists(
f"{closed_loop_rollout_data_path}.pkl"
):
rollout_data = collect_rollout_data(
env,
agent,
position_gridsize=position_gridsize,
angle_gridsize=angle_gridsize,
output_folder=output_folder,
output_prefix=f"closed_loop_${experiment_name}",
)
save_obj(rollout_data, closed_loop_rollout_data_path)
rollout_data = load_obj(closed_loop_rollout_data_path)
evaluated_rollouts = evaluate_rollout_data(env, rollout_data, ground_truth_brt)
plt = visualize_evaluated_rollout_stats(
evaluated_rollouts, title=f"{experiment_name} Closed-Loop Rollout Evaluation"
)
if generate_videos:
generate_representative_rollout_videos(env, evaluated_rollouts, output_folder, experiment_name)
if the_ipython_instance is not None and show_plots:
IPython.display.display(plt)
# ----------------------- open-loop rollout data collection
open_loop_rollout_data_path = os.path.join(
output_folder,
experiment_name + "open_loop_rollout_data",
)
if reproduce_open_loop_rollouts or not os.path.exists(
f"{open_loop_rollout_data_path}.pkl"
):
open_loop_rollout_data = collect_rollout_data(
env,
agent,
position_gridsize=position_gridsize,
angle_gridsize=angle_gridsize,
output_folder=output_folder,
output_prefix=f"open_loop_${experiment_name}",
enable_observation_feedback=False,
)
save_obj(open_loop_rollout_data, open_loop_rollout_data_path)
open_loop_rollout_data = load_obj(open_loop_rollout_data_path)
evaluated_open_loop_rollouts = evaluate_rollout_data(
env, open_loop_rollout_data, ground_truth_brt
)
plt = visualize_evaluated_rollout_stats(
evaluated_open_loop_rollouts,
title=f"{experiment_name} Open-Loop Rollout Evaluation",
)
if generate_videos:
generate_representative_rollout_videos(env, evaluated_open_loop_rollouts, output_folder, f"open_loop_{experiment_name}")
if the_ipython_instance is not None and show_plots:
IPython.display.display(plt)
In [ ]:
position_gridsize = 10
angle_gridsize = 3
default_config = RARL_wm.get_config(parse_args=False)
default_env, default_environment_info = RARL_wm.construct_environment(
default_config, visualize_failure_sets=False
)
agent = load_best_agent(default_config, default_environment_info)
default_ground_truth_brt = np.load(default_config.grid_path)
--------------------- Experiment name: 1129/084949 Task: dubins Logging to: logs/dreamer_dubins/1129/084949 --------------------- == Environment Information == dubins_car_img-v1 <module 'safety_rl.gym_reachability.gym_reachability' from '/home/lassepe/worktree/latent-safety/safety_rl/gym_reachability/gym_reachability/__init__.py'> Remove dubins_car-v1 from registry Remove dubins_car_img-v1 from registry Remove dubins_car_img_cont-v1 from registry config.obs_x: 0 config.obs_y: 0 config.obs_r: 0.5 Env: mode-RA; doneType-toEnd; sample_inside_obs-True Encoder CNN shapes: {'image': (128, 128, 3)} Encoder MLP shapes: {'obs_state': (2,)} Decoder CNN shapes: {'image': (128, 128, 3)} Decoder MLP shapes: {'obs_state': (2,)} Optimizer model_opt has 27134213 variables. logs/experiments/car-DDQN/world_model_new-toEnd State Dimension: 544, ActionSpace Dimension: 3 Dynamic parameters: CAR Constraint: 0.5 Turn: 0.80 Max speed: 1.00 Max angular speed: 1.250 ENV Constraint: 0.5 Turn: 0.80 Max speed: 1.00 [-1.25 0. 1.25] Type II Reach-Avoid Set == Agent Information == ModuleList( (0): Linear(in_features=544, out_features=100, bias=True) (1): Tanh() (2): Linear(in_features=100, out_features=100, bias=True) (3): Tanh() (4): Linear(in_features=100, out_features=3, bias=True) (5): Identity() ) DDQN: mode-RA; terminalType-g We want to use: cuda, and Agent uses: cuda Critic is using cuda: True => Restore /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/model/Q-150000.pth
In [ ]:
experiment_setups = {}
# -------------------------------------------------------appearance ood setups
# in-distribution setup
experiment_setups["nominal"] = {
"env": default_env,
"ground_truth_brt": default_ground_truth_brt,
}
# out-of-distribution evaluation with cyan background
experiment_setups["cyanbg_ood"] = {
"env": RARL_wm.construct_environment(
default_config, visualize_failure_sets=False, ood_dict={"background": (0, 1, 1)}
)[0],
"ground_truth_brt": default_ground_truth_brt,
}
# out-of-distribution evaluation with magenta background
experiment_setups["magentabg_ood"] = {
"env": RARL_wm.construct_environment(
default_config, visualize_failure_sets=False, ood_dict={"background": (1, 0, 1)}
)[0],
"ground_truth_brt": default_ground_truth_brt,
}
# out-of-distribution evaluation with different scale
experiment_setups["scaled01_ood"] = {
"env": RARL_wm.construct_environment(
default_config, visualize_failure_sets=False, ood_dict={"scale": 0.1}
)[0],
"ground_truth_brt": default_ground_truth_brt,
}
# out-of-distribution evaluation with magenta obstacles
experiment_setups["magentaobs_ood"] = {
"env": RARL_wm.construct_environment(
default_config, visualize_failure_sets=False, ood_dict={"obstacle_color": (1, 0, 1)}
)[0],
"ground_truth_brt": default_ground_truth_brt,
}
# out-of-distribution evaluation with yellow obstacles
experiment_setups["yellowobs_ood"] = {
"env": RARL_wm.construct_environment(
default_config, visualize_failure_sets=False, ood_dict={"obstacle_color": (1, 1, 0)}
)[0],
"ground_truth_brt": default_ground_truth_brt,
}
# ------------------------------------------------------- position ood setups
# out-of-distriution with offset x position
offsetx_ood_config = RARL_wm.get_config(parse_args=False, root_key="offsetx_ood")
offsetx_ood_env, _ = RARL_wm.construct_environment(
offsetx_ood_config, visualize_failure_sets=False
)
offsetx_ood_brt = np.load(offsetx_ood_config.grid_path)
experiment_setups["offsetx_ood"] = {
"env": offsetx_ood_env,
"ground_truth_brt": offsetx_ood_brt,
}
offsety_ood_config = RARL_wm.get_config(parse_args=False, root_key="offsety_ood")
offsety_ood_env, _ = RARL_wm.construct_environment(
offsety_ood_config, visualize_failure_sets=False
)
offsety_ood_brt = np.load(offsety_ood_config.grid_path)
experiment_setups["offsety_ood"] = {
"env": offsety_ood_env,
"ground_truth_brt": offsety_ood_brt,
}
offsetr_ood_config = RARL_wm.get_config(parse_args=False, root_key="offsetr_ood")
offsetr_ood_env, _ = RARL_wm.construct_environment(
offsetr_ood_config, visualize_failure_sets=False
)
offsetr_ood_brt = np.load(offsetr_ood_config.grid_path)
experiment_setups["offsetr_ood"] = {
"env": offsetr_ood_env,
"ground_truth_brt": offsetr_ood_brt,
}
== Environment Information ==
dubins_car_img-v1
<module 'safety_rl.gym_reachability.gym_reachability' from '/home/lassepe/worktree/latent-safety/safety_rl/gym_reachability/gym_reachability/__init__.py'>
config.obs_x: 0
config.obs_y: 0
config.obs_r: 0.5
Env: mode-RA; doneType-toEnd; sample_inside_obs-True
Encoder CNN shapes: {'image': (128, 128, 3)}
Encoder MLP shapes: {'obs_state': (2,)}
Decoder CNN shapes: {'image': (128, 128, 3)}
Decoder MLP shapes: {'obs_state': (2,)}
Optimizer model_opt has 27134213 variables.
logs/experiments/car-DDQN/world_model_new-toEnd
State Dimension: 544, ActionSpace Dimension: 3
Dynamic parameters:
CAR
Constraint: 0.5 Turn: 0.80 Max speed: 1.00 Max angular speed: 1.250
ENV
Constraint: 0.5 Turn: 0.80 Max speed: 1.00
[-1.25 0. 1.25]
Type II Reach-Avoid Set
== Environment Information ==
dubins_car_img-v1
<module 'safety_rl.gym_reachability.gym_reachability' from '/home/lassepe/worktree/latent-safety/safety_rl/gym_reachability/gym_reachability/__init__.py'>
config.obs_x: 0
config.obs_y: 0
config.obs_r: 0.5
Env: mode-RA; doneType-toEnd; sample_inside_obs-True
Encoder CNN shapes: {'image': (128, 128, 3)}
Encoder MLP shapes: {'obs_state': (2,)}
Decoder CNN shapes: {'image': (128, 128, 3)}
Decoder MLP shapes: {'obs_state': (2,)}
Optimizer model_opt has 27134213 variables.
logs/experiments/car-DDQN/world_model_new-toEnd
State Dimension: 544, ActionSpace Dimension: 3
Dynamic parameters:
CAR
Constraint: 0.5 Turn: 0.80 Max speed: 1.00 Max angular speed: 1.250
ENV
Constraint: 0.5 Turn: 0.80 Max speed: 1.00
[-1.25 0. 1.25]
Type II Reach-Avoid Set
== Environment Information ==
dubins_car_img-v1
<module 'safety_rl.gym_reachability.gym_reachability' from '/home/lassepe/worktree/latent-safety/safety_rl/gym_reachability/gym_reachability/__init__.py'>
config.obs_x: 0
config.obs_y: 0
config.obs_r: 0.5
Env: mode-RA; doneType-toEnd; sample_inside_obs-True
Encoder CNN shapes: {'image': (128, 128, 3)}
Encoder MLP shapes: {'obs_state': (2,)}
Decoder CNN shapes: {'image': (128, 128, 3)}
Decoder MLP shapes: {'obs_state': (2,)}
Optimizer model_opt has 27134213 variables.
logs/experiments/car-DDQN/world_model_new-toEnd
State Dimension: 544, ActionSpace Dimension: 3
Dynamic parameters:
CAR
Constraint: 0.5 Turn: 0.80 Max speed: 1.00 Max angular speed: 1.250
ENV
Constraint: 0.5 Turn: 0.80 Max speed: 1.00
[-1.25 0. 1.25]
Type II Reach-Avoid Set
== Environment Information ==
dubins_car_img-v1
<module 'safety_rl.gym_reachability.gym_reachability' from '/home/lassepe/worktree/latent-safety/safety_rl/gym_reachability/gym_reachability/__init__.py'>
config.obs_x: 0
config.obs_y: 0
config.obs_r: 0.5
Env: mode-RA; doneType-toEnd; sample_inside_obs-True
Encoder CNN shapes: {'image': (128, 128, 3)}
Encoder MLP shapes: {'obs_state': (2,)}
Decoder CNN shapes: {'image': (128, 128, 3)}
Decoder MLP shapes: {'obs_state': (2,)}
Optimizer model_opt has 27134213 variables.
logs/experiments/car-DDQN/world_model_new-toEnd
State Dimension: 544, ActionSpace Dimension: 3
Dynamic parameters:
CAR
Constraint: 0.5 Turn: 0.80 Max speed: 1.00 Max angular speed: 1.250
ENV
Constraint: 0.5 Turn: 0.80 Max speed: 1.00
[-1.25 0. 1.25]
Type II Reach-Avoid Set
== Environment Information ==
dubins_car_img-v1
<module 'safety_rl.gym_reachability.gym_reachability' from '/home/lassepe/worktree/latent-safety/safety_rl/gym_reachability/gym_reachability/__init__.py'>
config.obs_x: 0
config.obs_y: 0
config.obs_r: 0.5
Env: mode-RA; doneType-toEnd; sample_inside_obs-True
Encoder CNN shapes: {'image': (128, 128, 3)}
Encoder MLP shapes: {'obs_state': (2,)}
Decoder CNN shapes: {'image': (128, 128, 3)}
Decoder MLP shapes: {'obs_state': (2,)}
Optimizer model_opt has 27134213 variables.
logs/experiments/car-DDQN/world_model_new-toEnd
State Dimension: 544, ActionSpace Dimension: 3
Dynamic parameters:
CAR
Constraint: 0.5 Turn: 0.80 Max speed: 1.00 Max angular speed: 1.250
ENV
Constraint: 0.5 Turn: 0.80 Max speed: 1.00
[-1.25 0. 1.25]
Type II Reach-Avoid Set
---------------------
Experiment name: 1129/084952
Task: dubins
Logging to: logs/dreamer_dubins/1129/084952
---------------------
== Environment Information ==
dubins_car_img-v1
<module 'safety_rl.gym_reachability.gym_reachability' from '/home/lassepe/worktree/latent-safety/safety_rl/gym_reachability/gym_reachability/__init__.py'>
config.obs_x: 0.5
config.obs_y: 0.0
config.obs_r: 0.5
Env: mode-RA; doneType-toEnd; sample_inside_obs-True
Encoder CNN shapes: {'image': (128, 128, 3)}
Encoder MLP shapes: {'obs_state': (2,)}
Decoder CNN shapes: {'image': (128, 128, 3)}
Decoder MLP shapes: {'obs_state': (2,)}
Optimizer model_opt has 27134213 variables.
logs/experiments/car-DDQN/world_model_new-toEnd
State Dimension: 544, ActionSpace Dimension: 3
Dynamic parameters:
CAR
Constraint: 0.5 Turn: 0.80 Max speed: 1.00 Max angular speed: 1.250
ENV
Constraint: 0.5 Turn: 0.80 Max speed: 1.00
[-1.25 0. 1.25]
Type II Reach-Avoid Set
---------------------
Experiment name: 1129/084952
Task: dubins
Logging to: logs/dreamer_dubins/1129/084952
---------------------
== Environment Information ==
dubins_car_img-v1
<module 'safety_rl.gym_reachability.gym_reachability' from '/home/lassepe/worktree/latent-safety/safety_rl/gym_reachability/gym_reachability/__init__.py'>
config.obs_x: 0.0
config.obs_y: 0.5
config.obs_r: 0.5
Env: mode-RA; doneType-toEnd; sample_inside_obs-True
Encoder CNN shapes: {'image': (128, 128, 3)}
Encoder MLP shapes: {'obs_state': (2,)}
Decoder CNN shapes: {'image': (128, 128, 3)}
Decoder MLP shapes: {'obs_state': (2,)}
Optimizer model_opt has 27134213 variables.
logs/experiments/car-DDQN/world_model_new-toEnd
State Dimension: 544, ActionSpace Dimension: 3
Dynamic parameters:
CAR
Constraint: 0.5 Turn: 0.80 Max speed: 1.00 Max angular speed: 1.250
ENV
Constraint: 0.5 Turn: 0.80 Max speed: 1.00
[-1.25 0. 1.25]
Type II Reach-Avoid Set
---------------------
Experiment name: 1129/084953
Task: dubins
Logging to: logs/dreamer_dubins/1129/084953
---------------------
== Environment Information ==
dubins_car_img-v1
<module 'safety_rl.gym_reachability.gym_reachability' from '/home/lassepe/worktree/latent-safety/safety_rl/gym_reachability/gym_reachability/__init__.py'>
config.obs_x: 0.0
config.obs_y: 0.0
config.obs_r: 1.0
Env: mode-RA; doneType-toEnd; sample_inside_obs-True
Encoder CNN shapes: {'image': (128, 128, 3)}
Encoder MLP shapes: {'obs_state': (2,)}
Decoder CNN shapes: {'image': (128, 128, 3)}
Decoder MLP shapes: {'obs_state': (2,)}
Optimizer model_opt has 27134213 variables.
logs/experiments/car-DDQN/world_model_new-toEnd
State Dimension: 544, ActionSpace Dimension: 3
Dynamic parameters:
CAR
Constraint: 1.0 Turn: 0.80 Max speed: 1.00 Max angular speed: 1.250
ENV
Constraint: 1.0 Turn: 0.80 Max speed: 1.00
[-1.25 0. 1.25]
Type II Reach-Avoid Set
In [ ]:
# run all of the experiment setups in sequence
for experiment_name, experiment_setup in experiment_setups.items():
print(f"Running evaluation for {experiment_name}")
img = Image.fromarray(experiment_setup["env"].capture_image())
if the_ipython_instance is not None:
IPython.display.display(img)
evaluate(
env=experiment_setup["env"],
agent=agent,
ground_truth_brt=experiment_setup["ground_truth_brt"],
experiment_name=experiment_name,
position_gridsize=position_gridsize,
angle_gridsize=angle_gridsize,
reproduce_closed_loop_rollouts=False,
reproduce_open_loop_rollouts=False,
reproduce_value_function=False,
)
Running evaluation for nominal
tn: 0.216 tp: 0.757 fn: 0.024 fp: 0.003 accuracy: 0.973 precision: 0.995 recall: 0.970 f1: 0.982 Generating video for safe rollout 0
<ipython-input-2-a7a296c4c91a>:212: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at /home/conda/feedstock_root/build_artifacts/libtorch_1715185017593/work/torch/csrc/utils/tensor_new.cpp:261.)
Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/nominal_safe_0.mp4 Generating video for safe rollout 1 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/nominal_safe_1.mp4 Generating video for safe rollout 2 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/nominal_safe_2.mp4 Generating video for safe rollout 3 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/nominal_safe_3.mp4 Generating video for safe rollout 5 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/nominal_safe_5.mp4 Generating video for unsafe rollout 4 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/nominal_unsafe_4.mp4 Generating video for unsafe rollout 14 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/nominal_unsafe_14.mp4 Generating video for unsafe rollout 15 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/nominal_unsafe_15.mp4 Generating video for unsafe rollout 24 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/nominal_unsafe_24.mp4 Generating video for unsafe rollout 25 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/nominal_unsafe_25.mp4
Generating video for safe rollout 0 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/open_loop_nominal_safe_0.mp4 Generating video for safe rollout 1 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/open_loop_nominal_safe_1.mp4 Generating video for safe rollout 2 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/open_loop_nominal_safe_2.mp4 Generating video for safe rollout 3 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/open_loop_nominal_safe_3.mp4 Generating video for safe rollout 5 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/open_loop_nominal_safe_5.mp4 Generating video for unsafe rollout 4 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/open_loop_nominal_unsafe_4.mp4 Generating video for unsafe rollout 6 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/open_loop_nominal_unsafe_6.mp4 Generating video for unsafe rollout 14 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/open_loop_nominal_unsafe_14.mp4 Generating video for unsafe rollout 15 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/open_loop_nominal_unsafe_15.mp4 Generating video for unsafe rollout 24 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/nominal/open_loop_nominal_unsafe_24.mp4
Running evaluation for cyanbg_ood
tn: 0.220 tp: 0.000 fn: 0.780 fp: 0.000 accuracy: 0.220 precision: 0.000 recall: 0.000 f1: 0.000 Generating video for safe rollout 0 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/cyanbg_ood_safe_0.mp4 Generating video for safe rollout 1 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/cyanbg_ood_safe_1.mp4 Generating video for safe rollout 2 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/cyanbg_ood_safe_2.mp4 Generating video for safe rollout 3 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/cyanbg_ood_safe_3.mp4 Generating video for safe rollout 4 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/cyanbg_ood_safe_4.mp4 Generating video for unsafe rollout 14 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/cyanbg_ood_unsafe_14.mp4 Generating video for unsafe rollout 15 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/cyanbg_ood_unsafe_15.mp4 Generating video for unsafe rollout 23 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/cyanbg_ood_unsafe_23.mp4 Generating video for unsafe rollout 24 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/cyanbg_ood_unsafe_24.mp4 Generating video for unsafe rollout 25 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/cyanbg_ood_unsafe_25.mp4
Generating video for safe rollout 0 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/open_loop_cyanbg_ood_safe_0.mp4 Generating video for safe rollout 1 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/open_loop_cyanbg_ood_safe_1.mp4 Generating video for safe rollout 2 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/open_loop_cyanbg_ood_safe_2.mp4 Generating video for safe rollout 3 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/open_loop_cyanbg_ood_safe_3.mp4 Generating video for safe rollout 4 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/open_loop_cyanbg_ood_safe_4.mp4 Generating video for unsafe rollout 5 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/open_loop_cyanbg_ood_unsafe_5.mp4 Generating video for unsafe rollout 6 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/open_loop_cyanbg_ood_unsafe_6.mp4 Generating video for unsafe rollout 7 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/open_loop_cyanbg_ood_unsafe_7.mp4 Generating video for unsafe rollout 8 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/open_loop_cyanbg_ood_unsafe_8.mp4 Generating video for unsafe rollout 14 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/cyanbg_ood/open_loop_cyanbg_ood_unsafe_14.mp4
Running evaluation for magentabg_ood
tn: 0.220 tp: 0.000 fn: 0.780 fp: 0.000 accuracy: 0.220 precision: 0.000 recall: 0.000 f1: 0.000 Generating video for safe rollout 0 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/magentabg_ood_safe_0.mp4 Generating video for safe rollout 1 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/magentabg_ood_safe_1.mp4 Generating video for safe rollout 2 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/magentabg_ood_safe_2.mp4 Generating video for safe rollout 3 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/magentabg_ood_safe_3.mp4 Generating video for safe rollout 4 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/magentabg_ood_safe_4.mp4 Generating video for unsafe rollout 14 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/magentabg_ood_unsafe_14.mp4 Generating video for unsafe rollout 15 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/magentabg_ood_unsafe_15.mp4 Generating video for unsafe rollout 23 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/magentabg_ood_unsafe_23.mp4 Generating video for unsafe rollout 24 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/magentabg_ood_unsafe_24.mp4 Generating video for unsafe rollout 25 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/magentabg_ood_unsafe_25.mp4
Generating video for safe rollout 0 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/open_loop_magentabg_ood_safe_0.mp4 Generating video for safe rollout 1 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/open_loop_magentabg_ood_safe_1.mp4 Generating video for safe rollout 2 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/open_loop_magentabg_ood_safe_2.mp4 Generating video for safe rollout 3 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/open_loop_magentabg_ood_safe_3.mp4 Generating video for safe rollout 4 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/open_loop_magentabg_ood_safe_4.mp4 Generating video for unsafe rollout 5 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/open_loop_magentabg_ood_unsafe_5.mp4 Generating video for unsafe rollout 6 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/open_loop_magentabg_ood_unsafe_6.mp4 Generating video for unsafe rollout 7 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/open_loop_magentabg_ood_unsafe_7.mp4 Generating video for unsafe rollout 8 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/open_loop_magentabg_ood_unsafe_8.mp4 Generating video for unsafe rollout 14 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentabg_ood/open_loop_magentabg_ood_unsafe_14.mp4
Running evaluation for scaled01_ood
tn: 0.208 tp: 0.738 fn: 0.042 fp: 0.012 accuracy: 0.946 precision: 0.984 recall: 0.946 f1: 0.965 Generating video for safe rollout 0 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/scaled01_ood_safe_0.mp4 Generating video for safe rollout 1 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/scaled01_ood_safe_1.mp4 Generating video for safe rollout 2 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/scaled01_ood_safe_2.mp4 Generating video for safe rollout 3 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/scaled01_ood_safe_3.mp4 Generating video for safe rollout 5 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/scaled01_ood_safe_5.mp4 Generating video for unsafe rollout 4 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/scaled01_ood_unsafe_4.mp4 Generating video for unsafe rollout 14 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/scaled01_ood_unsafe_14.mp4 Generating video for unsafe rollout 15 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/scaled01_ood_unsafe_15.mp4 Generating video for unsafe rollout 23 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/scaled01_ood_unsafe_23.mp4 Generating video for unsafe rollout 24 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/scaled01_ood_unsafe_24.mp4
Generating video for safe rollout 0 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/open_loop_scaled01_ood_safe_0.mp4 Generating video for safe rollout 1 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/open_loop_scaled01_ood_safe_1.mp4 Generating video for safe rollout 2 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/open_loop_scaled01_ood_safe_2.mp4 Generating video for safe rollout 3 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/open_loop_scaled01_ood_safe_3.mp4 Generating video for safe rollout 5 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/open_loop_scaled01_ood_safe_5.mp4 Generating video for unsafe rollout 4 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/open_loop_scaled01_ood_unsafe_4.mp4 Generating video for unsafe rollout 14 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/open_loop_scaled01_ood_unsafe_14.mp4 Generating video for unsafe rollout 15 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/open_loop_scaled01_ood_unsafe_15.mp4 Generating video for unsafe rollout 23 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/open_loop_scaled01_ood_unsafe_23.mp4 Generating video for unsafe rollout 24 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/scaled01_ood/open_loop_scaled01_ood_unsafe_24.mp4
Running evaluation for magentaobs_ood
tn: 0.220 tp: 0.000 fn: 0.780 fp: 0.000 accuracy: 0.220 precision: 0.000 recall: 0.000 f1: 0.000 Generating video for safe rollout 0 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/magentaobs_ood_safe_0.mp4 Generating video for safe rollout 1 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/magentaobs_ood_safe_1.mp4 Generating video for safe rollout 2 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/magentaobs_ood_safe_2.mp4 Generating video for safe rollout 3 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/magentaobs_ood_safe_3.mp4 Generating video for safe rollout 4 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/magentaobs_ood_safe_4.mp4 Generating video for unsafe rollout 5 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/magentaobs_ood_unsafe_5.mp4 Generating video for unsafe rollout 14 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/magentaobs_ood_unsafe_14.mp4 Generating video for unsafe rollout 15 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/magentaobs_ood_unsafe_15.mp4 Generating video for unsafe rollout 23 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/magentaobs_ood_unsafe_23.mp4 Generating video for unsafe rollout 24 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/magentaobs_ood_unsafe_24.mp4
Generating video for safe rollout 0 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/open_loop_magentaobs_ood_safe_0.mp4 Generating video for safe rollout 1 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/open_loop_magentaobs_ood_safe_1.mp4 Generating video for safe rollout 2 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/open_loop_magentaobs_ood_safe_2.mp4 Generating video for safe rollout 3 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/open_loop_magentaobs_ood_safe_3.mp4 Generating video for safe rollout 8 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/open_loop_magentaobs_ood_safe_8.mp4 Generating video for unsafe rollout 4 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/open_loop_magentaobs_ood_unsafe_4.mp4 Generating video for unsafe rollout 5 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/open_loop_magentaobs_ood_unsafe_5.mp4 Generating video for unsafe rollout 6 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/open_loop_magentaobs_ood_unsafe_6.mp4 Generating video for unsafe rollout 7 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/open_loop_magentaobs_ood_unsafe_7.mp4 Generating video for unsafe rollout 14 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/magentaobs_ood/open_loop_magentaobs_ood_unsafe_14.mp4
Running evaluation for yellowobs_ood
tn: 0.213 tp: 0.764 fn: 0.016 fp: 0.006 accuracy: 0.978 precision: 0.992 recall: 0.980 f1: 0.986 Generating video for safe rollout 0 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/yellowobs_ood_safe_0.mp4 Generating video for safe rollout 1 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/yellowobs_ood_safe_1.mp4 Generating video for safe rollout 2 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/yellowobs_ood_safe_2.mp4 Generating video for safe rollout 3 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/yellowobs_ood_safe_3.mp4 Generating video for safe rollout 5 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/yellowobs_ood_safe_5.mp4 Generating video for unsafe rollout 4 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/yellowobs_ood_unsafe_4.mp4 Generating video for unsafe rollout 14 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/yellowobs_ood_unsafe_14.mp4 Generating video for unsafe rollout 15 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/yellowobs_ood_unsafe_15.mp4 Generating video for unsafe rollout 24 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/yellowobs_ood_unsafe_24.mp4 Generating video for unsafe rollout 25 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/yellowobs_ood_unsafe_25.mp4
Generating video for safe rollout 0 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/open_loop_yellowobs_ood_safe_0.mp4 Generating video for safe rollout 1 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/open_loop_yellowobs_ood_safe_1.mp4 Generating video for safe rollout 2 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/open_loop_yellowobs_ood_safe_2.mp4 Generating video for safe rollout 3 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/open_loop_yellowobs_ood_safe_3.mp4 Generating video for safe rollout 5 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/open_loop_yellowobs_ood_safe_5.mp4 Generating video for unsafe rollout 4 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/open_loop_yellowobs_ood_unsafe_4.mp4 Generating video for unsafe rollout 6 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/open_loop_yellowobs_ood_unsafe_6.mp4 Generating video for unsafe rollout 14 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/open_loop_yellowobs_ood_unsafe_14.mp4 Generating video for unsafe rollout 15 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/open_loop_yellowobs_ood_unsafe_15.mp4 Generating video for unsafe rollout 24 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/yellowobs_ood/open_loop_yellowobs_ood_unsafe_24.mp4
Running evaluation for offsetx_ood
tn: 0.124 tp: 0.673 fn: 0.116 fp: 0.088 accuracy: 0.796 precision: 0.885 recall: 0.853 f1: 0.868 Generating video for safe rollout 2 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/offsetx_ood_safe_2.mp4 Generating video for safe rollout 3 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/offsetx_ood_safe_3.mp4 Generating video for safe rollout 4 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/offsetx_ood_safe_4.mp4 Generating video for safe rollout 7 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/offsetx_ood_safe_7.mp4 Generating video for safe rollout 8 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/offsetx_ood_safe_8.mp4 Generating video for unsafe rollout 0 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/offsetx_ood_unsafe_0.mp4 Generating video for unsafe rollout 1 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/offsetx_ood_unsafe_1.mp4 Generating video for unsafe rollout 5 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/offsetx_ood_unsafe_5.mp4 Generating video for unsafe rollout 6 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/offsetx_ood_unsafe_6.mp4 Generating video for unsafe rollout 10 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/offsetx_ood_unsafe_10.mp4
Generating video for safe rollout 0 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/open_loop_offsetx_ood_safe_0.mp4 Generating video for safe rollout 1 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/open_loop_offsetx_ood_safe_1.mp4 Generating video for safe rollout 2 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/open_loop_offsetx_ood_safe_2.mp4 Generating video for safe rollout 3 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/open_loop_offsetx_ood_safe_3.mp4 Generating video for safe rollout 4 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/open_loop_offsetx_ood_safe_4.mp4 Generating video for unsafe rollout 5 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/open_loop_offsetx_ood_unsafe_5.mp4 Generating video for unsafe rollout 6 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/open_loop_offsetx_ood_unsafe_6.mp4 Generating video for unsafe rollout 9 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/open_loop_offsetx_ood_unsafe_9.mp4 Generating video for unsafe rollout 11 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/open_loop_offsetx_ood_unsafe_11.mp4 Generating video for unsafe rollout 13 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetx_ood/open_loop_offsetx_ood_unsafe_13.mp4
Running evaluation for offsety_ood
tn: 0.083 tp: 0.625 fn: 0.156 fp: 0.136 accuracy: 0.708 precision: 0.821 recall: 0.800 f1: 0.810 Generating video for safe rollout 0 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/offsety_ood_safe_0.mp4 Generating video for safe rollout 1 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/offsety_ood_safe_1.mp4 Generating video for safe rollout 2 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/offsety_ood_safe_2.mp4 Generating video for safe rollout 3 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/offsety_ood_safe_3.mp4 Generating video for safe rollout 8 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/offsety_ood_safe_8.mp4 Generating video for unsafe rollout 4 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/offsety_ood_unsafe_4.mp4 Generating video for unsafe rollout 5 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/offsety_ood_unsafe_5.mp4 Generating video for unsafe rollout 6 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/offsety_ood_unsafe_6.mp4 Generating video for unsafe rollout 7 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/offsety_ood_unsafe_7.mp4 Generating video for unsafe rollout 15 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/offsety_ood_unsafe_15.mp4
Generating video for safe rollout 0 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/open_loop_offsety_ood_safe_0.mp4 Generating video for safe rollout 1 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/open_loop_offsety_ood_safe_1.mp4 Generating video for safe rollout 2 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/open_loop_offsety_ood_safe_2.mp4 Generating video for safe rollout 3 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/open_loop_offsety_ood_safe_3.mp4 Generating video for safe rollout 10 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/open_loop_offsety_ood_safe_10.mp4 Generating video for unsafe rollout 4 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/open_loop_offsety_ood_unsafe_4.mp4 Generating video for unsafe rollout 5 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/open_loop_offsety_ood_unsafe_5.mp4 Generating video for unsafe rollout 6 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/open_loop_offsety_ood_unsafe_6.mp4 Generating video for unsafe rollout 7 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/open_loop_offsety_ood_unsafe_7.mp4 Generating video for unsafe rollout 8 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsety_ood/open_loop_offsety_ood_unsafe_8.mp4
Running evaluation for offsetr_ood
tn: 0.236 tp: 0.299 fn: 0.000 fp: 0.465 accuracy: 0.535 precision: 0.391 recall: 1.000 f1: 0.562 Collecting rollouts for theta = 0 degrees; 1/3 Collecting rollouts for theta = 120 degrees; 2/3 Collecting rollouts for theta = 240 degrees; 3/3 Generating video for safe rollout 8 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/offsetr_ood_safe_8.mp4 Generating video for safe rollout 9 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/offsetr_ood_safe_9.mp4 Generating video for safe rollout 29 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/offsetr_ood_safe_29.mp4 Generating video for safe rollout 39 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/offsetr_ood_safe_39.mp4 Generating video for safe rollout 49 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/offsetr_ood_safe_49.mp4 Generating video for unsafe rollout 0 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/offsetr_ood_unsafe_0.mp4 Generating video for unsafe rollout 1 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/offsetr_ood_unsafe_1.mp4 Generating video for unsafe rollout 2 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/offsetr_ood_unsafe_2.mp4 Generating video for unsafe rollout 3 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/offsetr_ood_unsafe_3.mp4 Generating video for unsafe rollout 4 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/offsetr_ood_unsafe_4.mp4
Collecting rollouts for theta = 0 degrees; 1/3 Collecting rollouts for theta = 120 degrees; 2/3 Collecting rollouts for theta = 240 degrees; 3/3 Generating video for safe rollout 0 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/open_loop_offsetr_ood_safe_0.mp4 Generating video for safe rollout 1 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/open_loop_offsetr_ood_safe_1.mp4 Generating video for safe rollout 10 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/open_loop_offsetr_ood_safe_10.mp4 Generating video for safe rollout 20 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/open_loop_offsetr_ood_safe_20.mp4 Generating video for safe rollout 30 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/open_loop_offsetr_ood_safe_30.mp4 Generating video for unsafe rollout 2 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/open_loop_offsetr_ood_unsafe_2.mp4 Generating video for unsafe rollout 3 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/open_loop_offsetr_ood_unsafe_3.mp4 Generating video for unsafe rollout 4 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/open_loop_offsetr_ood_unsafe_4.mp4 Generating video for unsafe rollout 5 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/open_loop_offsetr_ood_unsafe_5.mp4 Generating video for unsafe rollout 6 Video saved as /home/lassepe/worktree/latent-safety/logs/experiments/car-DDQN/world_model_new-toEnd/offsetr_ood/open_loop_offsetr_ood_unsafe_6.mp4
